# plotly standard imports
import plotly.graph_objs as go
import chart_studio.plotly as py

# Cufflinks wrapper on plotly
import cufflinks

# Data science imports
import pandas as pd
import numpy as np

# Options for pandas
pd.options.display.max_columns = 30

# Display all cell outputs
from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = 'all'

from plotly.offline import iplot, init_notebook_mode
cufflinks.go_offline(connected=True)
init_notebook_mode(connected=True)

# Set global theme
cufflinks.set_config_file(world_readable=True, theme='pearl')

Let's explore datasets

Explore input dataset

Will use target dataset Bitcoin Historical Data

Bitcoin data at 1-min intervals from select exchanges, Jan 2012 to Dec 2020

from src.load_datasets import load_input_dataset

input_dataset = load_input_dataset()

input_dataset.head()

	Timestamp	Open	High	Low	Close	Volume_(BTC)	Volume_(Currency)	Weighted_Price
0	1325317920	4.39	4.39	4.39	4.39	0.455581	2.0	4.39
1	1325317980	NaN	NaN	NaN	NaN	NaN	NaN	NaN
2	1325318040	NaN	NaN	NaN	NaN	NaN	NaN	NaN
3	1325318100	NaN	NaN	NaN	NaN	NaN	NaN	NaN
4	1325318160	NaN	NaN	NaN	NaN	NaN	NaN	NaN

Will explore full input dataset, some values contain NaN, which not ineraptebale by sweetviz, so will use timestamp as target feature for now

import sweetviz as sv

analyse_report = sv.analyze([input_dataset, 'Input'], target_feat="Timestamp")
analyse_report.show_notebook()

Will take one timestamp per hour for faster interpretation

hours_dataset = input_dataset[59::60]

timestamp need interprate as date for charts processing

raw_timestamps = hours_dataset.pop('Timestamp')
hours_datetime = pd.to_datetime(raw_timestamps, unit='s')

hours_dataset.head()

	Open	High	Low	Close	Volume_(BTC)	Volume_(Currency)	Weighted_Price
59	NaN	NaN	NaN	NaN	NaN	NaN	NaN
119	NaN	NaN	NaN	NaN	NaN	NaN	NaN
179	NaN	NaN	NaN	NaN	NaN	NaN	NaN
239	NaN	NaN	NaN	NaN	NaN	NaN	NaN
299	NaN	NaN	NaN	NaN	NaN	NaN	NaN

Feature evalution over time

hours_features = hours_dataset[['Open', 'Close', 'Weighted_Price', 'Volume_(BTC)', 'Volume_(Currency)']]
hours_features.index = hours_datetime

hours_features.iplot(
    subplots=True,
)

hours_dataset.describe().transpose()

	count	mean	std	min	25%	50%	75%	max
Open	57539.0	4597.869997	4781.860481	4.230000	436.000000	3308.83000	8178.790000	2.892734e+04
High	57539.0	4600.855412	4785.273937	4.230000	436.335000	3312.82000	8183.195000	2.897090e+04
Low	57539.0	4594.739766	4778.247471	4.230000	435.900000	3304.00000	8173.460000	2.892734e+04
Close	57539.0	4597.868472	4781.911473	4.230000	436.025000	3305.00000	8177.300000	2.897090e+04
Volume_(BTC)	57539.0	9.003106	29.133884	0.000001	0.370362	1.85132	6.922594	1.540919e+03
Volume_(Currency)	57539.0	31184.490106	105810.113064	0.000178	411.516416	3109.00990	20379.086506	4.865723e+06
Weighted_Price	57539.0	4597.865954	4781.891626	4.230000	436.030107	3305.86258	8177.593176	2.895104e+04

Will take only last three yers, because they have data without missing values

day = 24
year = (365)*day

years_count = 3.5
items_count = round(years_count * year)

last_years_dataset = hours_dataset[-1 * items_count:]
last_years_datetime = hours_datetime[-1 * items_count:]

last_years_dataset.head()
len(last_years_dataset)

	Open	High	Low	Close	Volume_(BTC)	Volume_(Currency)	Weighted_Price
2888219	2429.91	2430.00	2429.90	2430.00	0.342051	831.155079	2429.915723
2888279	2420.20	2423.69	2420.20	2423.69	0.605780	1466.827998	2421.385540
2888339	2438.42	2438.42	2438.42	2438.42	0.027000	65.837340	2438.420000
2888399	2437.00	2437.00	2437.00	2437.00	1.057586	2577.337813	2437.000000
2888459	2466.25	2466.25	2465.98	2465.98	7.223139	17813.647056	2466.192025

30660

last_years_features = last_years_dataset[['Open', 'Close', 'Weighted_Price', 'Volume_(BTC)', 'Volume_(Currency)']]
last_years_features.index = last_years_datetime

last_years_features.iplot(
    subplots=True,
)

Remove NaN values

# Is have NaN
last_years_dataset.isnull().values.any()

True

last_years_dataset.isnull().sum().sum()

6825

last_years_dataset_with_time = input_dataset[59::60][-1 * items_count:]
len(last_years_dataset_with_time)
len(last_years_dataset)

nan_datafreame = last_years_dataset_with_time[last_years_dataset_with_time.isna().any(axis=1)]
nan_datafreame

30660

30660

	Timestamp	Open	High	Low	Close	Volume_(BTC)	Volume_(Currency)	Weighted_Price
2888999	1499046180	NaN	NaN	NaN	NaN	NaN	NaN	NaN
2889479	1499074980	NaN	NaN	NaN	NaN	NaN	NaN	NaN
2890139	1499114580	NaN	NaN	NaN	NaN	NaN	NaN	NaN
2890799	1499154180	NaN	NaN	NaN	NaN	NaN	NaN	NaN
2890979	1499164980	NaN	NaN	NaN	NaN	NaN	NaN	NaN
...	...	...	...	...	...	...	...	...
4641599	1604202180	NaN	NaN	NaN	NaN	NaN	NaN	NaN
4643279	1604302980	NaN	NaN	NaN	NaN	NaN	NaN	NaN
4660559	1605339780	NaN	NaN	NaN	NaN	NaN	NaN	NaN
4662479	1605454980	NaN	NaN	NaN	NaN	NaN	NaN	NaN
4700879	1607758980	NaN	NaN	NaN	NaN	NaN	NaN	NaN

975 rows × 8 columns

nan_hours_datetime = pd.to_datetime(nan_datafreame.pop('Timestamp'), unit='s')

nan_features =  nan_datafreame['Weighted_Price']
nan_features.index = nan_hours_datetime
nan_features.iplot(
    kind='scatter',
    mode='markers'
)

Check depenence of trading and price from date in year and time of day

Firstly define function for display frequiency

import tensorflow as tf
import matplotlib.pyplot as plt

def plot_log_freaquency(series):
    fft = tf.signal.rfft(series)    
    f_per_dataset = np.arange(0, len(fft))

    n_samples_h = len(series)
    hours_per_year = 24*365.2524
    years_per_dataset = n_samples_h/(hours_per_year)

    f_per_year = f_per_dataset/years_per_dataset
    plt.step(f_per_year, np.abs(fft))
    plt.xscale('log')
    plt.xticks([1, 365.2524], labels=['1/Year', '1/day'])
    _ = plt.xlabel('Frequency (log scale)')

Frequency of price

without_nan = last_years_dataset.dropna()

plot_log_freaquency(without_nan['Weighted_Price'])

Frequency of transaction volume

plot_log_freaquency(without_nan['Volume_(Currency)'])

Training data distribution

train_df = pd.DataFrame(tfds.as_numpy(train_data), columns=['text', 'type'])

train_df['type'] = train_df['type'].apply(humanize_label)

train_df.head()

print('Training dataset records', len(train_df.index))

train_df['type'].iplot(
    kind='hist',
    yTitle='count',
    xTitle='Type',
    title='Training data distribution'
)

Testing data distribution

test_df = pd.DataFrame(tfds.as_numpy(test_data), columns=['text', 'type'])

test_df['type'] = test_df['type'].apply(humanize_label)

test_df[30:40]

print('Testing dataset records', len(test_df.index))

neutralSeries = test_df.apply(lambda x: True if x['type'] == 'neutral' else False, axis=1)
print('Count of neutral rows', len(neutralSeries[neutralSeries == True].index))

test_df['type'].iplot(
    kind='hist',
    yTitle='count',
    xTitle='Type',
    title='Testing data distribution'
)

Check preprocessed training datasets distribution

train_prep_df = pd.DataFrame(tfds.as_numpy(train_prep_dataset), columns=['text', 'type'])

train_prep_df['type'] = train_prep_df['type'].apply(humanize_label)

train_prep_df.head()

print('Training dataset records', len(train_prep_df.index))

train_prep_df['type'].iplot(
    kind='hist',
    yTitle='count',
    xTitle='Type',
    title='Preprocessed training data distribution'
)

Check testing dataset

test_prep_df = pd.DataFrame(tfds.as_numpy(test_prep_dataset), columns=['text', 'type'])

test_prep_df['type'] = test_prep_df['type'].apply(humanize_label)

test_prep_df.head()

print('Training dataset records', len(test_prep_df.index))

test_prep_df['type'].iplot(
    kind='hist',
    yTitle='count',
    xTitle='Type',
    title='Preprocessed testing data distribution'
)

Explore training metrics

df = pd.read_csv('./metrics/training.csv')
df.head()

df[['epoch', 'accuracy', 'val_accuracy']].iplot(
    x='epoch',
    mode='lines+markers',
    xTitle='epoch',
    yTitle='accuracy', 
    title='Training accuracy',
    linecolor='black',
)

df[['epoch', 'loss', 'val_loss']].iplot(
    x='epoch',
    mode='lines+markers',
    xTitle='epoch',
    yTitle='accuracy', 
    title='Losses'
)

Predictions

Load probability model

which can give predictions on model classes

0 - bad review, 1 - good revie

from src.predict import get_probability_model

model = get_probability_model()

Firstly will try predict on some data from training dataset

from src.predict import get_text_and_label_from_dataset, predict
REVIEW_INDEX = 110

text, real_label = get_text_and_label_from_dataset(REVIEW_INDEX)

print('text for prediction\n\n', text, '\n')

predicted_label, predictions = predict(text, model)

print(label_categories[predicted_label], 'review')

print('\n\nPredicted label:', predicted_label, 'real label: ', real_label, 'predictions:', predictions)
if (predicted_label == real_label):
    print('Successfully predicted')
else:
    print('Failed to predict')

Then will try predict hadnwritten text

# Can change text and check model
hadwriten = 'This is good film'

print('Hendwriten text:\n', hadwriten, '\n')

handwriten_label, predictions = predict(hadwriten, model)

print(label_categories[predicted_label], 'review')

print('Probabilities', predictions)